package com.qen.ktpython
// 多线程爬取同济新闻 news.tongji.edu.cn
import com.qen.ktpython.LinkProcessor.Companion.concatUrl
import com.qen.ktpython.LinkProcessor.Companion.isNewsPage
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withLock
import org.jsoup.Connection
import org.jsoup.Jsoup
import java.io.File
import java.io.PrintWriter
import java.util.concurrent.atomic.AtomicInteger
import org.jsoup.nodes.Document

/**
 * 网页信息文件写入工具。
 */
private class PageFileWrite(filePath: String) {
    private val file: File
    private val fileMutes = Mutex()

    private var printWriter: PrintWriter? = null

    init {
        file = File(filePath)
    }

    fun prepare() = apply {
        if (printWriter == null) {
            printWriter = file.printWriter()
        }
    }

    fun close() {
        printWriter?.close()
        printWriter = null
    }

    fun writeLine(line: String, divider: String? = "\n") = runBlocking {
        fileMutes.withLock {
            printWriter?.write(line)
            if (divider != null) {
                printWriter?.write(divider)
            }
            printWriter?.flush()
        }
    }

}

private class LinkStore {
    /** 读写锁。 */
    private val operationMutex = Mutex()

    /** 待爬取链接。 */
    private val linkQueue = ArrayList<String>()

    /** 已探索链接。 */
    private val linkDiscovered = HashSet<String>()
    fun put(link: String) = runBlocking {
        operationMutex.withLock {
            if (!linkDiscovered.contains(link)) {
                linkQueue.add(link)
                linkDiscovered.add(link)
            }
        }
    }

    fun get(): String? = runBlocking {
        var ret: String?
        operationMutex.withLock {
            ret = linkQueue.removeFirstOrNull()
        }
        return@runBlocking ret
    }
}

/**
 * 链接处理工具。
 */
private class LinkProcessor {
    companion object {
        /**
         * 链接是否指向新闻页面。
         */
        fun String.isNewsPage(): Boolean {
            return this.startsWith("https://news.tongji.edu.cn/info/")
        }

        /**
         * 合并双斜线。将 // 转换为 / （https:// 除外）。
         */
        fun String.mergeDoubleSlashes(): String {
            return this.replace("//", "/").replace("https:/", "https://")
        }

        /**
         * 拼接链接。
         * 如：https://tongji.edu.cn + info -> https://tongji.edu.cn/info
         *
         * 拼接过程会自动处理 ./ 和 ../ 这种怪怪的东西。
         */
        fun String.concatUrl(url: String): String {
            val parentSegments = this.substring("https://".length, this.length).split("/")
            val parentBuilder = StringBuilder("https://")

            var siteRootAttached = false
            if (parentSegments.size == 1) {
                parentBuilder.append(parentSegments[0])
            } else {
                parentSegments.forEach { seg ->
                    if (seg == parentSegments.last()) {
                        return@forEach
                    } else if (!siteRootAttached) {
                        parentBuilder.append(seg)
                        siteRootAttached = true
                    } else {
                        parentBuilder.append("/$seg")
                    }
                }
            }
            val realParent = parentBuilder.toString()
            var pureUrl = "$realParent/$url".mergeDoubleSlashes()
            pureUrl = pureUrl.substring("https://".length, pureUrl.length)

            val sourceUrlSegments = pureUrl.split("/")
            val resultUrlSegments = ArrayList<String>()

            sourceUrlSegments.forEach { segment ->
                if (segment == ".") {
                    return@forEach
                } else if (segment == "..") {
                    if (resultUrlSegments.isNotEmpty()) {
                        resultUrlSegments.removeLast()
                    }
                } else {
                    resultUrlSegments.add(segment)
                }
            }

            val resultBuilder = StringBuilder("https://")
            siteRootAttached = false
            resultUrlSegments.forEach { seg ->
                if (seg.isBlank()) {
                    return@forEach
                } else if (!siteRootAttached) {
                    resultBuilder.append(seg)
                    siteRootAttached = true
                } else {
                    resultBuilder.append("/$seg")
                }
            }
            return resultBuilder.toString()
        }
    }
}

/**
 * 新闻信息。
 */
private data class NewsData(
    var url: String = "",
    var source: String = "",
    var date: String = "",
    var title: String = "",
    var content: String = "",
) {
    companion object {
        private const val DIVIDER = "\n"
//        private const val DIVIDER = MacroDefines.NEWS_DATA_DIVIDER
    }

    override fun toString(): String {
        return "$url$DIVIDER$source$DIVIDER$date$DIVIDER$title$DIVIDER$content"
    }
}

/**爬虫目标网页根路径。同样为进入点。*/
private const val SPIDER_WEB_ROOT = "https://news.tongji.edu.cn"

/**并行爬虫线程数*/
private const val THREAD_COUNT = 9

/**需要爬取d额新闻数量*/
private const val TARGET_NEWS_COUNT = 100

/**用于识别href的正则表达式*/
private val hrefRegex = "href=\".*?\"".toRegex()

/**不感兴趣的链接后缀*/
private val pageExtensionIgnoreList = listOf(".png", ".jpg", ".mp4", ".css")

/*不感兴趣的 href 前缀 。**/
private val pagePrefixIgnoreList = listOf("https://", "http://", "mailto:", "ftp://", "#")

/**不感兴趣的 href 内容关键字*/
private val pageContentIgnoreKeywords =
    listOf("https://", "mailto:", "ftp://", "http://", "download.jsp", "javascript:")

fun main(args: Array<String>) {
    /**爬虫线程队列*/
    val threads = ArrayList<Thread>()

    /**文件写入器，创建后立即准备*/
    val fileWriter = PageFileWrite("./result1.txt").prepare()

    /**链接管理器*/
    val linkStore = LinkStore()

    /**已爬取的新闻数（所谓“已爬取”，实际是从拿到链接就算）*/
    val newsFetchedCounter = AtomicInteger()

    /**爬取结束信号*/
    val newsTargetReachedSemaphore = Semaphore(THREAD_COUNT, THREAD_COUNT)

    /**待爬取队列非空信号*/
    val linkQueueAvailableSemaphore = Semaphore(Int.MAX_VALUE, Int.MAX_VALUE)
    /**将首页加入进去*/
    linkStore.put(SPIDER_WEB_ROOT)
    /**放出一个信号*/
    linkQueueAvailableSemaphore.release()
    /**创建并启动爬虫线程*/
    for (i in 0 until THREAD_COUNT) {
        val thread = Thread {

            /**-------------  爬虫爬取线程内容---开始---------------  */
            while (true) {
                runBlocking {
                    linkQueueAvailableSemaphore.acquire()  // 获取信号
                }

                if (newsFetchedCounter.get() >= TARGET_NEWS_COUNT) { //如果拿到目标数就结束
                    break
                }

                val link = linkStore.get() ?: continue   //  拿链接
                println("${Thread.currentThread().id}:fetching $link")

                /**连接网站*/
                val response: Connection.Response = Jsoup.connect(link)
                    .followRedirects(false)
                    .execute()

                /**检查状态码*/
                val statusCode = response.statusCode()

                if (statusCode != 200) {
                    println("${Thread.currentThread().id} : error while fetching $link. code is $statusCode")
                    continue
                }

                /**获取网页文本*/
                val document = response.parse()

                if (link.isNewsPage()) {
                    /**------------ 新闻页面 ： 读取内容 -------------- */
                    // 提取内容
                    val newsData = document.toNewsData()
                    if (newsData != null) {
                        val newsCount = newsFetchedCounter.incrementAndGet()

                        println("${Thread.currentThread().id} : news $newsCount : $link")

                        if (newsCount <= TARGET_NEWS_COUNT) {
                            newsData.url = link
                            fileWriter.writeLine(newsData.toString())
                        }

                        if (newsCount >= TARGET_NEWS_COUNT) {
                            newsTargetReachedSemaphore.release()
                        }
                    }
                }
                /**------------------------所有页面：读取并处理所有的href -------------------*/

                val regexResults = hrefRegex.findAll(document.toString())

                println(regexResults);
                regexResults.forEach regexResult@{
                    val href = it.value.substring("href=\"".length, it.value.length - 1)
                    println("it.value:${it.value}")
                    /**不看含不感兴趣的关键字的。*/
                    pageContentIgnoreKeywords.forEach { t ->
                        if (href.contains(t)) {
                            return@regexResult
                        }
                    }
                    /**不看前缀不对的，如mailto:*/
                    pagePrefixIgnoreList.forEach { t ->
                        if (href.startsWith(t)) {
                            return@regexResult
                        }
                    }
                    /**不看后缀不对的。如jpg*/
                    pageExtensionIgnoreList.forEach { t ->
                        if (href.endsWith(t)) {
                            return@regexResult
                        }
                    }
                    linkStore.put(link.concatUrl(href))
                    linkQueueAvailableSemaphore.release()
                }
            }
            /**-------------  爬虫爬取线程内容---结束---------------  */
        }
        thread.start()
        threads.add(thread)
    }

    runBlocking {
        newsTargetReachedSemaphore.acquire()
    }
    /**等待所有爬虫工作完毕*/
    threads.forEach { t -> t.join() }

    /**关闭文件输出流*/
    fileWriter.close()

}


private fun Document.toNewsData(): NewsData? {
    val ret = NewsData()

    val content = this.getElementsByClass("content").first() ?: return null

    /**获取标题*/
    val titleElement = content.getElementsByTag("h3").first() ?: return null
    ret.title = titleElement.text()

    /**来源和时间*/
    val sourceAndDateContainer = content.getElementsByTag("i").first() ?: return null
    val sourceAndDateSegments = sourceAndDateContainer.text()
        .split("浏览").first()
        .split("时间：")

    ret.source = sourceAndDateSegments[0].split("：")[1]
    ret.date = sourceAndDateSegments[1]

    val newsContentContainer = content.getElementsByClass("v_news_content").first() ?: return null
    ret.content = newsContentContainer.text()
        .replace("\n", "  ")
        .replace("\r", "  ")
    return if (ret.title.isNotBlank() && ret.content.isNotBlank() && ret.date.isNotBlank()) {
        ret
    } else {
        null
    }
}






